package spark.mllib.synonyms

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.mllib.stat.Statistics
import org.apache.spark.sql.SparkSession

/**
  * Created by liuwei on 2017/9/6.
  */
object SimilarityUtil {

  def main(args: Array[String]): Unit = {

    val v1 = Vector(1.0, 3.0, 5.0)
    val v2 = Vector(2.0, 6.0, 10.0)
    val v3 = Vector(4.0, 12.0, 20.0)
    val v4 = Vector(2.0, 4.0, 6.0)
    val v5 = Vector(3.0, 5.0, 7.0)
    val v6 = Vector(0.0, 0.0, 0.0)
    val v7 = Seq(3.0, 2010.0000000001)
    val v8 = Seq(3.0, 2010.0000000002)
    val sparkConf = new SparkConf().setAppName("LDATest").setMaster("local[8]")
    val sc = new SparkContext(sparkConf)
    val rdd01 = sc.makeRDD(v7)
    val rdd02 = sc.makeRDD(v8)
    println(Statistics.corr(rdd01, rdd02))
    //    println(cosineSimilarity(v7, v8))
    //    println(cosineSimilarity(v1, v2))
    //    println(cosineSimilarity(v1, v3))
    //    println(cosineSimilarity(v1, v6))
    //    println(cosineSimilarity(v6, v6))
    //    println(cosineSimilarity(v6, v1))
    println(L2Similarity(v1, v2))
    println(L2Similarity(v1, v4))
    println(L2Similarity(v4, v5))
    println(L2Similarity(v1, v6))
    println(L1Similarity(v1, v2))
    println(L1Similarity(v1, v4))
    println(L1Similarity(v4, v5))
    println(L1Similarity(v1, v6))
  }


  /**
    * 计算两个vector的cos相似度
    * temp1 参数1的平方和开根号
    * temp2 参数2的平方和开根号
    */
  def cosineSimilarity(vector1: scala.Seq[Double], vector2: scala.Seq[Double], temp1: Double, temp2: Double): Double = {
    val member = vector1.zip(vector2).map(d => d._1 * d._2).sum //对公式分子部分进行计算
    val denominator = temp1 * temp2 //求出分母
    if (0 == denominator) return 0.0
    if (member > denominator) 1.0 else member / denominator //进行计算
  }

  /**
    * 计算两个vector的L2相似度(欧式距离 欧几里得度量)
    */
  def L2Similarity(vector1: scala.Seq[Double], vector2: scala.Seq[Double]): Double = {
    1.0 / (1.0 + Math.sqrt(vector1.zip(vector2).map(d => Math.pow(d._1 - d._2, 2)).sum))
  }

  /**
    * 计算两个vector的L1相似度(曼哈顿距离)
    */
  def L1Similarity(vector1: scala.Seq[Double], vector2: scala.Seq[Double]): Double = {
    1.0 / (1.0 + vector1.zip(vector2).map(d => Math.abs(d._1 - d._2)).sum)
  }

}
